package com.spark;


import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import okhttp3.*;
import org.apache.commons.io.FileUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;


/**
 * Created by TRS on 2017/6/9.
 */
public class ES {


    public static OkHttpClient client = new OkHttpClient();

    public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20);


    public static void main(String[] args) throws Exception {
        getNews();
        //getDoc();




    }

    public static void getDoc() throws IOException {
        File file = new File("F:\\data\\news\\21530000.json");
        List<String> lines = FileUtils.readLines(file, "UTF-8");

        List<String> zjolNews = new ArrayList<String>();

        for (String line : lines) {
            JSONObject jsonObject = JSON.parseObject(line);
            String id = jsonObject.getString("newsid");
            String title = jsonObject.getString("title");
            String content = getTextFromTHML(jsonObject.getString("source"));
            if (content.length() > 400) {
                zjolNews.add(id.substring(1, id.length()) + "\t" + content + "\t" + title);
            }
        }

        File zjolNewsFile = new File("F:\\data\\zjolnews.txt");
        FileUtils.writeLines(zjolNewsFile, zjolNews);


    }


    public static void getNews() {
        final String fileRoot = "F:\\data\\news\\";


        int files = 1;
        final List<String> lines = new ArrayList<String>();
        //021531820
        Integer id = 21550000;
        while (id > 21540000) {
            final Integer nId = id;
            fixedThreadPool.submit(new Runnable() {
                public void run() {
                    try {
                        int end = nId - 10000;
                        List<String> news = new ArrayList<String>();
                        for (int subId = nId; subId > end; subId--) {
                            String newId = "0" + subId;
                            String json = getJson(newId);

                            if (!json.equals("nocontent")) {
                                try {
                                    JSONObject okJosn = JSON.parseObject(json);
                                    news.add(okJosn.toJSONString());
                                    System.out.println(nId + " " + newId + " " + news.size());
                                } catch (Exception e) {
                                    e.printStackTrace();
                                }

                            }
                        }
                        File f = new File(fileRoot + nId + ".json");
                        FileUtils.writeLines(f, "UTF-8", news, true);


                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            });
            id -= 10000;

        }
    }

    public static void toes() {
        String url = "http://e.zjol.com/news/zjol/";

        Integer id = 21561820;
        while (id > 21560000) {

            final Integer nId = id;
            final String nUrl = url;

            fixedThreadPool.submit(new Runnable() {
                public void run() {
                    try {
                        String newId = "0" + nId;
                        String json = getJson(newId);
                        if (!json.equals("nocontent")) {
                            index(nUrl + newId, json);
                        } else {
                            System.out.println("nocontent");
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            });

            id--;

        }
    }


    public static void index(String url, String json) throws IOException {
        MediaType JSON = MediaType.parse("application/json; charset=utf-8");

        RequestBody body = RequestBody.create(JSON, json);
        Request request = new Request.Builder()
                .url(url)
                .put(body)
                .build();
        Response response = client.newCall(request).execute();

        System.out.println(response.toString());
    }

    public static String getJson(String newID) {
        String newUrl = "http://m.zjol.com.cn/mzjol/api?action=getnews&newsid=" + newID;

        System.out.println(newUrl);
        String json = null;
        Request request = new Request.Builder()
                .url(newUrl)
                .get()
                .build();
        Response response = null;
        try {
            response = client.newCall(request).execute();
            json = response.body().string();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return json;
    }

    private static String getTextFromTHML(String html) {
        Document doc = Jsoup.parse(html);
        String text1 = doc.text();
        // remove extra white space
        StringBuilder builder = new StringBuilder(text1);
        int index = 0;
        while (builder.length() > index) {
            Character tmp = builder.charAt(index);
            if (Character.isSpaceChar(tmp) || Character.isWhitespace(tmp)) builder.setCharAt(index, ' ');
            index += 1;
        }
        text1 = builder.toString().replaceAll(" +", " ").trim();

        return text1;
    }

}
